import numpy as np

import pandas as pd

import missingno 

import plotly 

from plotly import offline

from plotly import graph_objs as go

from plotly import express as px

import seaborn as sns

sns.set(color_codes=True)

import matplotlib.pyplot as plt

​

​

import datetime

from scipy.stats import chi2_contingency, beta 

from IPython.display import Image

import statsmodels.stats.api as sms

from math import ceil

import scipy.stats as stats

df=pd.read_csv("converted_data.csv")

df.sample(5)

print("There are {:,} observations and {} columns in the data set.".format(df.shape[0], df.shape[1]))

df.columns 

df.dtypes

df.describe()

missingno.bar(df,color="#AFE8BB",fontsize=30)

C = (df.dtypes == 'object')

CategoricalVariables = list(C[C].index)

​

Integer = (df.dtypes == 'int64') 

Float   = (df.dtypes == 'float64') 

NumericVariables = list(Integer[Integer].index) + list(Float[Float].index)

​

Missing_Percentage = (df.isnull().sum()).sum()/np.product(df.shape)*100

print("The number of missing entries before cleaning: " + str(round(Missing_Percentage,2)) + " %")

plt.boxplot(df["age"])

plt.show()

index = df[(df['age'] >= 100)].index

df.drop(index, inplace=True)

df['age'].describe()

#checking the null values

df.isnull().sum()

#removing the null values

df.dropna(axis="rows", how="any", inplace = True)

df.reset_index(drop=True, inplace=True)

missingno.bar(df,color="#AFE8BB",fontsize=30)

#checking the null values after cleaning 

df.isnull().sum()

df.describe().T

df.select_dtypes(include=['object']).describe()

#Preparing for visualisation by using Pie shape. It represents data visually as a fractional part of a whole.

​

def create_pie(name,title,text):

    """ creating Pie chart using plotly with 3 argument name,title,text"""

    labels=name.index

    values=name.values

    trace0=go.Pie(values=values,labels=labels,hole=0.4,textposition="inside",textinfo="label+percent+value")

    data=[trace0]

    layout=dict(title=title,title_x=0.5,annotations=[dict(text=text,x=0.5,y=0.5,showarrow=False,font_size=14)])

    fig=dict(data=data,layout=layout)

    offline.iplot(fig)

#Preparing for visualisation by using sunburst shape.Sunburst Visualization is ideal for displaying hierarchical data. The visualization design is made up of an inner circle surrounded by rings of deeper hierarchy levels. The angle of each segment is either proportional to a value or divided equally under its parent node.

​

def create_sunburst(df,path:list,values,title):

    """ Creating sunburst chart using plotly python"""

    fig=px.sunburst(df,path=path,values=values)

    fig.update_layout(title=title,title_x=0.5)

    fig.show()

plt.boxplot(df["age"])

plt.show()

sns.displot(df['age'],color="#AFE8BB")

#Feature Engineering 

​

df.loc[(df["age"]>=10)&(df["age"]<=19),"in_age"]="in 10's"

df.loc[(df["age"]>=20)&(df["age"]<=29),"in_age"]="in 20's"

df.loc[(df["age"]>=30)&(df["age"]<=39),"in_age"]="in 30's"

df.loc[(df["age"]>=40)&(df["age"]<=49),"in_age"]="in 40's"

df.loc[(df["age"]>=50)&(df["age"]<=59),"in_age"]="in 50's"

df.loc[(df["age"]>=60)&(df["age"]<=69),"in_age"]="in 60's"

df.loc[(df["age"]>=70)&(df["age"]<=79),"in_age"]="in 70's"

df.loc[(df["age"]>=80)&(df["age"]<=89),"in_age"]="in 80's"

df.loc[(df["age"]>=90)&(df["age"]<=99),"in_age"]="in 90's"

df.sample(5)

df.info()

df.columns

df.describe()

#By product

df["first_product"].unique()

product=df["first_product"].value_counts(sort=True)

product

create_pie(product,"Product Distribution","Product") 

df.groupby("first_product").agg(Minimum_age=("age","min"),

                               Maximum_Age=("age","max"),

                               Average_Age=("age","mean")).reset_index()

# By Age

df["age"].min()

df["age"].max()

age=df.groupby("in_age").agg(Frequency=("in_age","count")).reset_index()

age

create_sunburst(age,["in_age","Frequency"],"Frequency",

                "Age Distribution")

product_vs_inage=df.groupby(["first_product","in_age"]).agg(Frequency=("first_product","count")).reset_index()

product_vs_inage

create_sunburst(product_vs_inage,["first_product","in_age","Frequency"],

                "Frequency","Product Age Frequency")

# By Region 

df["area_classification"].unique()

region=df["area_classification"].value_counts(sort=True)

region

create_pie(region,"Region Classification","Region")

df_product_Area=df['area_classification'].value_counts().to_frame().reset_index().rename(columns={'index':'area_classification','area_classification':'count'})

​

fig = go.Figure(data=[go.Scatter(

    x=df_product_Area['area_classification'], y=df_product_Area['count'],

    mode='markers',

    marker=dict(

        color=df_product_Area['count'],

        size=df_product_Area['count']*0.070, # Multiplying by 0.3 to reduce size and stay uniform accross all points

        showscale=True

    ))])

​

fig.update_layout(title='Area classification',xaxis_title="Regions",yaxis_title="Frequency Of Area ",title_x=0.5)

fig.show()

Product_vs_region=df.groupby(["area_classification","first_product"]).agg(Product_Count=("first_product","count")).reset_index()

Product_vs_region

create_sunburst(Product_vs_region,["area_classification","first_product","Product_Count"],

                "Product_Count","Product Region and Count.")

inage_vs_region=df.groupby(["first_product","in_age","area_classification"]).agg(frequency=("area_classification","count")).reset_index()

inage_vs_region

create_sunburst(inage_vs_region,["first_product","in_age","area_classification","frequency"]

                ,"frequency","Product age and region.")

fig = px.scatter(inage_vs_region, x='frequency', y='in_age', color='area_classification', size='frequency',

                 title="The frequency of the issued policies increases with rural and big 7 areas among 20's and 30's",

                 color_discrete_sequence=['#AFE8BB','#003C2B'],height=600)

fig.update_layout(legend=dict(title='',orientation="h", yanchor="bottom", y=1.02, xanchor="right", x=1),

                  font_color="#303030", xaxis=dict(title='Policies frequencies',showgrid=False), 

                  yaxis=dict(title='Age Group',showgrid=False, zerolinecolor='#E5E5EA',

                             showline=True, linecolor='#E5E5EA', linewidth=2))

fig.show()